#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2018/1/28 0028 14:18
# @Author  : Arliki
# @File    : comment.py

import json, re, scrapy, time
from scrapy_redis.spiders import RedisSpider
from onepiece.items import TotalItem

class TotalSpider(RedisSpider):
    name = "total_url"
    redis_key = "onepiece:total_url"
    custom_settings = {
        'REDIS_PARAMS': {
            'db': 1,
            'password': '5tgbnhy67ujm'
        }
    }

    def __init__(self, *args, **kwargs):
        domain = kwargs.pop("domain", "")
        self.allowed_domains = filter(None, domain.split(","))
        super(TotalSpider, self).__init__(*args, **kwargs)

    def parse(self, response):
        for u in response.xpath('//div[@id="content"]/li'):
            item = TotalItem()
            item['t_url'] = ''.join(response.url + u.xpath('a/@href')[0].extract())
            yield item
        # lpush onepiece:total_url http://www.fzdm.com/manhua/02/
